"""
Copyright (c) 2024 Huawei Technologies Co.,Ltd.

"""
"""
Case Type   : 资源池化
Case Name   : 资源池化场景下, 开启按需回放参数进行备节点failover操作，failover过程中备机又发生故障，重启后节点恢复正常（仅限2节点环境）
Create At   : 2024-05-07
Owner       : @chendong76
Description :
    1、查看集群状态
    2、修改相关参数，打开按需回放功能
    3、变更集群至2节点环境
    4、开始运行tpcc
    5、备机进行failover，sleep50s后，检查集群状态
    6、kill升主备机，等待CM重新拉起备机
    7、查看集群状态
    8、拉起剩余节点
    9、恢复环境为初始集群状态，参数恢复默认值
Expect      :
    1、集群状态正常
    2、参数修改成功，按需回放功能开启
    3、集群状态变更成功
    4、运行成功，此处不检查
    5、failover执行成功，可以查询到promoting标记
    6、执行成功
    7、查询集群状态成功，备机升主完成
    8、执行成功
    9、执行成功
History     :
"""

import os
import unittest
import time

from yat.test import Node
from testcase.utils.ComThread import ComThread
from testcase.utils.Common import Common
from testcase.utils.CommonSH import CommonSH
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from yat.test import macro

Primary_SH = CommonSH('PrimaryDbUser')
Primary_ROOTSH = CommonSH('PrimaryRoot')
Standby1_SH = CommonSH('Standby1DbUser')

@unittest.skipIf(1 == Primary_SH.get_node_num(), '单机环境不执行')
class SharedStorage(unittest.TestCase):

    def setUp(self):
        self.log = Logger()
        self.log.info(f'-----{os.path.basename(__file__)} start-----')
        self.constant = Constant()
        self.com = Common()

        self.s1_comsh = CommonSH('Standby1DbUser')
        self.s2_comsh = CommonSH('Standby2DbUser')
        self.common = Common('Standby1DbUser')
        self.tpcc_user = Node('PrimaryRoot')
        self.recovery_parse_workers = 'recovery_parse_workers'
        self.recovery_redo_workers = 'recovery_redo_workers'
        self.hot_standby = 'hot_standby'
        self.ss_enable_ondemand_recovery = 'ss_enable_ondemand_recovery'
        self.ss_enable_ondemand_realtime_build = 'ss_enable_ondemand_realtime_build'
        self.ss_ondemand_recovery_mem_size = 'ss_ondemand_recovery_mem_size'

        status = Primary_SH.exec_cm_ctl(mode='query', param='-Cv')
        if "cluster_state   : Normal" in status:
            self.log.info(status + "集群状态正常")
        else:
            stopdn = Primary_SH.exec_cm_ctl('stop')
            self.log.info(stopdn)
            status = Primary_SH.exec_cm_ctl(mode='query', param='-Cv')
            startdn = Primary_SH.exec_cm_ctl('start')
            self.log.info(startdn)
            self.assertIn(self.constant.cm_start_success_msg, startdn,
                    '拉起集群失败')
            status = Primary_SH.exec_cm_ctl(mode='query', param='-Cv')
            self.log.info(status)

        self.recovery_parse_workers_default = self.common.show_param(self.recovery_parse_workers)
        self.recovery_redo_workers_default = self.common.show_param(self.recovery_redo_workers)
        self.hot_standby_default = self.common.show_param(self.hot_standby)
        self.ss_enable_ondemand_recovery_default = self.common.show_param(self.ss_enable_ondemand_recovery)
        self.ss_enable_ondemand_realtime_build_default = self.common.show_param(self.ss_enable_ondemand_realtime_build)
        self.ss_ondemand_recovery_mem_size_default = self.common.show_param(self.ss_ondemand_recovery_mem_size)

    def test_shared_storage(self):
        text = '-----step1:查看集群状态; expect:修改成功-----'
        self.log.info(text)
        status = Primary_SH.exec_cm_ctl(mode='query', param='-Cv')
        self.log.info(status)
        self.assertIn('cluster_state   : Normal', status, '执行失败' + text)

        text = '-----step2:修改相关参数，打开按需回放功能; expect:执行成功-----'
        self.log.info(text)
        result = self.s1_comsh.execute_gsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            f'{self.recovery_parse_workers}=2')
        self.assertTrue(result, '执行失败' + text)
        result = self.s1_comsh.execute_gsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            f'{self.recovery_redo_workers}=2')
        self.assertTrue(result, '执行失败' + text)
        result = self.s1_comsh.execute_gsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            f'{self.hot_standby}=off')
        self.assertTrue(result, '执行失败' + text)
        result = self.s1_comsh.execute_gsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            f'{self.ss_enable_ondemand_recovery}=on')
        self.assertTrue(result, '执行失败' + text)
        result = self.s1_comsh.execute_gsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            f'{self.ss_enable_ondemand_realtime_build}=off')
        self.assertTrue(result, '执行失败' + text)
        result = self.s1_comsh.execute_gsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            f'{self.ss_ondemand_recovery_mem_size}=5GB')
        self.assertTrue(result, '执行失败' + text)
        restart_res = self.s1_comsh.restart_db_cluster()
        self.assertTrue(restart_res, '执行失败' + text)
        status = Primary_SH.exec_cm_ctl(mode='query', param='-Cv')
        self.log.info(status)
        text = '-----step3:调整至2节点环境; expect:执行成功-----'
        self.log.info(text)
        recovery = Primary_SH.exec_cm_ctl(mode='stop', param='-n 3')
        self.log.info(recovery)
        self.assertIn(self.constant.cm_stop_node_success_msg, recovery,
                      '执行失败' + text)
        status = Primary_SH.exec_cm_ctl(mode='query', param='-Cv')
        self.log.info(status)
        self.assertIn('cluster_state   : Degraded', status, '执行失败' + text)
        self.assertIn('P Primary Normal',
                      status.splitlines()[-1].split('|')[0].split('6001')[-1],
                      '执行失败' + text)
        self.assertIn('S Standby Normal',
                      status.splitlines()[-1].split('|')[1].split('6002')[-1],
                      '执行失败' + text)
        
        text = '-----step4:开始运行tpcc; expect:运行成功，此处不做检查-----'
        self.log.info(text)
        start_tpcc = ComThread(self.com.start_tpcc,
                                 args=(self.tpcc_user,'/data/benchmarksql-5.0/run',
                                       './runBenchmark.sh primary_write.pg'))
        start_tpcc.setDaemon(True)
        start_tpcc.start()
        status = Primary_ROOTSH.exec_cmd_under_root("ps -ux | grep './runBenchmark.sh' | grep -v -- '--color'")
        self.log.info(status)
        self.assertIn('primary_write.pg', status, '执行失败' + text)
        time.sleep(10)

        text = '-----step5:备机进行failover，sleep50s后，检查集群状态; expect:执行成功-----'
        self.log.info(text)
        stopdn = Primary_SH.exec_failover()
        self.log.info(stopdn)
        time.sleep(120)
        status = Primary_SH.exec_cm_ctl(mode='query', param='-Cv')
        res1 = status.splitlines()[-1].split('|')[0].split('6001')[-1]
        res2 = status.splitlines()[-1].split('|')[1].split('6002')[-1]
        res3 = status.splitlines()[-1].split('|')[2].split('6003')[-1]
        self.log.info(status)
        self.log.info(res1)
        self.log.info(res2)
        self.log.info(res3)
        self.assertIn('Down    Manually stopped', res1, '执行失败' + text)
        self.assertIn('Primary Normal', res2, '执行失败' + text)
        self.assertIn('Down    Unknown', res3, '执行失败' + text)

        text = '-----step6:kill升主备机，等待CM重新拉起备机; expect:执行成功-----'
        self.log.info(text)
        stopdn = Standby1_SH.exec_kill()
        self.log.info(stopdn)
        status = Standby1_SH.exec_cm_ctl(mode='query', param='-Cv')
        self.log.info(status)
        for i in range(0, 10):
            time.sleep(10)
            status = Primary_SH.exec_cm_ctl(mode='query', param='-Cv')
            self.log.info(status)
            if "Primary Normal" in status:
                break
            time.sleep(10)

        text = '-----step7:查看集群状态; expect:执行成功-----'
        self.log.info(text)
        status = Standby1_SH.exec_cm_ctl(mode='query', param='-Cv')
        self.log.info(status)
        self.assertIn('Primary Normal', status, '执行失败' + text)

        text = '-----step8:拉起其余节点; expect:执行成功-----'
        self.log.info(text)
        startdn = Primary_SH.exec_cm_ctl('start')
        self.log.info(startdn)
        self.assertIn(self.constant.cm_start_success_msg, startdn, '拉起dn节点失败')

    def tearDown(self):
        text = '-----step9:恢复环境为初始集群状态; expect:执行成功-----'
        self.log.info(text)
        status = Primary_SH.exec_cm_ctl(mode='query', param='-Cv')
        self.log.info(status)
        status = Primary_SH.check_cluster_core_status(status, wait_times=600)
        self.log.info(status)
        # 设置参数为默认值
        result = self.s1_comsh.execute_gsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            f'{self.recovery_parse_workers}={self.recovery_parse_workers_default}')
        self.assertTrue(result, '执行失败' + text)
        result = self.s1_comsh.execute_gsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            f'{self.recovery_redo_workers}={self.recovery_redo_workers_default}')
        self.assertTrue(result, '执行失败' + text)
        result = self.s1_comsh.execute_gsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            f'{self.hot_standby}={self.hot_standby_default}')
        self.assertTrue(result, '执行失败' + text)
        result = self.s1_comsh.execute_gsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            f'{self.ss_enable_ondemand_recovery}={self.ss_enable_ondemand_recovery_default}')
        self.assertTrue(result, '执行失败' + text)
        result = self.s1_comsh.execute_gsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            f'{self.ss_enable_ondemand_realtime_build}={self.ss_enable_ondemand_realtime_build_default}')
        self.assertTrue(result, '执行失败' + text)
        result = self.s1_comsh.execute_gsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            f'{self.ss_ondemand_recovery_mem_size}={self.ss_ondemand_recovery_mem_size_default}')
        self.assertTrue(result, '执行失败' + text)

        # 重启数据库，并执行switchover重分布节点角色
        restart_res = self.s1_comsh.restart_db_cluster()
        self.assertTrue(restart_res, '执行失败' + text)
        recovery = Primary_SH.exec_cm_ctl(mode='switchover', param='-a')
        self.log.info(recovery)
        self.assertIn(self.constant.cm_switchover_success_msg, recovery, '执行失败' + text)

        # 检查参数是否生效
        result_param = self.common.show_param(self.recovery_parse_workers)
        self.assertEquals(result_param, self.recovery_parse_workers_default, '执行失败' + text)
        result_param = self.common.show_param(self.recovery_redo_workers)
        self.assertEquals(result_param, self.recovery_redo_workers_default, '执行失败' + text)
        result_param = self.common.show_param(self.hot_standby)
        self.assertEquals(result_param, self.hot_standby_default, '执行失败' + text)
        result_param = self.common.show_param(self.ss_enable_ondemand_recovery)
        self.assertEquals(result_param, self.ss_enable_ondemand_recovery_default, '执行失败' + text)
        result_param = self.common.show_param(self.ss_enable_ondemand_realtime_build)
        self.assertEquals(result_param, self.ss_enable_ondemand_realtime_build_default, '执行失败' + text)
        result_param = self.common.show_param(self.ss_ondemand_recovery_mem_size)
        self.assertEquals(result_param, self.ss_ondemand_recovery_mem_size_default, '执行失败' + text)

        # 查询节点状态
        status = Primary_SH.exec_cm_ctl(mode='query', param='-Cv')
        self.log.info(status)
        self.assertIn('cluster_state   : Normal', status, '执行失败' + text)
        self.assertIn('P Primary Normal',
                      status.splitlines()[-1].split('|')[0].split('6001')[-1],
                      '执行失败' + text)
        self.assertIn('S Standby Normal',
                      status.splitlines()[-1].split('|')[1].split('6002')[-1],
                      '执行失败' + text)
        self.assertIn('S Standby Normal',
                      status.splitlines()[-1].split('|')[2].split('6003')[-1],
                      '执行失败' + text)
        self.log.info(f'-----{os.path.basename(__file__)} end-----')
